package com.dbs.zstp.wiki.engine;

import com.dbs.zstp.wiki.mapper.WqzbDataMapper;
import com.dbs.zstp.wiki.mapper.WqzbMapper;
import com.dbs.zstp.wiki.model.Wqzb;
import com.dbs.zstp.wiki.model.WqzbData;
import com.github.nobodxbodon.zhconverter.简繁转换类;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.File;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.*;

/**
 * 中文维基百科抓取引擎
 *
 * @author NPF
 * @create 2018-03-20 16:56
 **/
public class ZhWikiPageProcessor implements PageProcessor{

    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private WqzbMapper wqzbMapper;

    private WqzbDataMapper wqzbDataMapper;

    //当前爬虫关键字
    private String keyword="";

    //图片下载跟目录
    private String zhwiki_root_path = "";

    //图片路径
    private String img_path="";

    //图片对应File对象
    private File img_file = null;

    //图片对应的URL
    private String img_url;

    //图片宽度，
    int img_width = 0;

    //解析infoBox元素，下载图片
    List<String> imgs=null;

    //响应式图片srcset 参数
    String img_srcset = "";
    //响应式图片srcset 参数 对应多个地址
    String[] img_srcsets = null;

    private List<WqzbData> wqzbDatas;

    private Wqzb wqzb;

    private WqzbData wqzbData;

    private static ApplicationContext applicationContext;//启动类set入，调用下面set方法

    public static void setApplicationContext(ApplicationContext context) {
        applicationContext = context;
    }

    public ZhWikiPageProcessor() {

    }

    public ZhWikiPageProcessor(String zhwiki_root_path,String keyword) {
        this.zhwiki_root_path = zhwiki_root_path;
        this.keyword = keyword;

    }



    /**
     *  第一步：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
     */
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000)
                        .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36")
                        .setCharset("utf-8");

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
     * 1、根据维基百科分类页【中国武器、日本武器等】作为爬虫入口
     * 2、爬取分类信息后，添加到爬虫序列中
     * 3、如果是详情页面，则抓取数据放到数据库中
     * @param page
     */
    @Override
    public void process(Page page) {

        if(null == wqzbMapper || null == wqzbDataMapper){
            //设置图片存放根目录
            zhwiki_root_path +="zhwiki_img";
            wqzbMapper = applicationContext.getBean(WqzbMapper.class);
            wqzbDataMapper = applicationContext.getBean(WqzbDataMapper.class);
        }

        String fromurl = page.getUrl().toString();

        //判断当前页面是分类页面还是详情页面
        if(fromurl.indexOf("https://zh.wikipedia.org/wiki/Category")>-1){ //分类页面，不做解析，只是存放连接
            page.addTargetRequests(page.getHtml().$("#mw-subcategories a").links().all());
            page.addTargetRequests(page.getHtml().$("#mw-pages a").links().all());
        }else{ //详情页分析
            Wqzb wqzb = new Wqzb();
            wqzbDatas = new ArrayList<WqzbData>();


            String name = page.getHtml().xpath("//*[@id='firstHeading']/text()").toString();
            String text = Jsoup.parse(page.getHtml().$("#mw-content-text p").toString()).text();
            //获取父分类名称
            String normal_catlinks = "";
            List<String> categoryList = page.getHtml().$("#mw-normal-catlinks a[title^='Category']").all();
            List<String> categoryTextArray = new ArrayList<>();
            for (int i = 0; i <categoryList.size() ; i++) {
                categoryTextArray.add(Jsoup.parse(categoryList.get(i)).text());
            }
            normal_catlinks=String.join(",", categoryTextArray);

            fromurl = 简繁转换类.转换(fromurl,简繁转换类.目标.简体);
            name = 简繁转换类.转换(name,简繁转换类.目标.简体);
            text = 简繁转换类.转换(text,简繁转换类.目标.简体);
            normal_catlinks = 简繁转换类.转换(normal_catlinks,简繁转换类.目标.简体);

            // 第二步：定义如何抽取页面信息，并保存下来
            page.putField("source","中文维基百科");
            page.putField("fromurl",fromurl);
            page.putField("name",name);
            page.putField("text",text);

            // 根据不同的页面规则，抽取infobox属性
            Map<String,String> keyValMap = new HashMap<String,String>();
            String key = "",val="";

            List<String> tr_html=null,th_keys = null,td_vals=null;
            th_keys = page.getHtml().$("table.infobox tr th[scope='row']").all();
            if(th_keys.size()>0){ //infobox 的key字段使用 th[scope='row'] 表示
                td_vals = page.getHtml().$("table.infobox tr th[scope='row'] +td").all();

                //th 是key,td是value
                Document key_doc = null;
                Document val_doc = null;

                for (int i = 0; i < th_keys.size(); i++) {
                    key_doc = Jsoup.parse(th_keys.get(i));
                    val_doc = Jsoup.parse(td_vals.get(i));
                    key = key_doc.text();
                    val = val_doc.text();
                    if(StringUtils.isNotBlank(key) && StringUtils.isNotBlank(val)){
                        key = 简繁转换类.转换(key,简繁转换类.目标.简体);
                        val = 简繁转换类.转换(val,简繁转换类.目标.简体);
                        keyValMap.put(key,val);
                    }
                }
            }else{ //infobox 的key、val全部用 td表示
                tr_html = page.getHtml().$("table.infobox tr").all();
                Document tr_html_doc = null;
                for (int i = 0; i < tr_html.size(); i++) {
                    tr_html_doc = Jsoup.parse(tr_html.get(i));

                    key = tr_html_doc.select("b").text();
                    val = tr_html_doc.text().replace(key,"");
                    if(StringUtils.isNotBlank(key) && StringUtils.isNotBlank(val)){
                        key = 简繁转换类.转换(key,简繁转换类.目标.简体);
                        val = 简繁转换类.转换(val,简繁转换类.目标.简体);
                        keyValMap.put(key,val);
                    }
                }

            }

            page.putField("json", keyValMap);

            //获取当前所有图片
            imgs = page.getHtml().$(".mw-parser-output .image").all();

            if (imgs.size()>0){
                try {
                    for (int i = 0; i <imgs.size() ; i++) {
                        img_width = Integer.parseInt(Jsoup.parse(imgs.get(i)).select("img").attr("width"));
                        if(img_width>200){ //只有图片大于200px的，才有下载价值
                            img_srcset = Jsoup.parse(imgs.get(i)).select("img").attr("srcset");
                            img_srcsets = img_srcset.split(", ");
                            img_url = img_srcsets[img_srcsets.length-1].split(" ")[0];
                            wqzb.setImg_url(img_url);
                            page.putField("img_path", downloadImg(img_url));
                            wqzb.setImg_path(img_path);
                            break;
                        }
                    }

                } catch (Exception e) {
                    logger.error(name+"-----图片下载失败!-----url"+img_srcset);
                    e.printStackTrace();
                }

            }

            //武器装备主表信息保存
            wqzb.setId(UUID.randomUUID().toString());
            wqzb.setName(name);
            wqzb.setJs(text);
            wqzb.setUrl(fromurl);
            wqzb.setType(normal_catlinks);
            wqzb.setKeyword(keyword);
            wqzbMapper.insertSelective(wqzb);


            //武器装备子表信息保存
            if(StringUtils.isNotBlank(wqzb.getId())){
                Set<String> keySets = keyValMap.keySet();
                for (String k : keySets ) {
                    wqzbData = new WqzbData();
                    wqzbData.setWqzb_id(wqzb.getId());
                    wqzbData.setWqzb_key(k);
                    wqzbData.setWqzb_val(keyValMap.get(k));
                    wqzbDataMapper.insertSelective(wqzbData);
                }


            }
        }

    }


    public String downloadImg(String imgURL) throws Exception {
        //随机产生100之内整数
        int random = (int)(Math.random()*100)+1;
        String suffix = imgURL.substring(imgURL.lastIndexOf("."),imgURL.length());
        img_path = zhwiki_root_path +File.separator+random+File.separator+ UUID.randomUUID()+suffix;
        //img地址随机分配到100之内的地址内
        img_file = new File(img_path);
        URL url = new URL("https:"+imgURL);
        URLConnection uc = url.openConnection();
        InputStream out = uc.getInputStream();

        FileUtils.copyInputStreamToFile(out,img_file);
        return img_path;
    }


}
